import os,sys
sys.path.append(r"D:\\source\\work\\spiders")

from xxqg.db import *
from xxqg.spider import spider
from xxqg.utils import *
from urllib import parse
import time
import datetime
from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
import shutil
import traceback
import re

ztspider = spider('https://www.xuexi.cn/ba07e0e0ca3b0302dcdc364b6449b5a8/b96df2c4f343da84b4d59835eee51f61.html')


def run(index):
    basedir = r'\\192.168.10.60\d$\chuanshu\数据采集[张俊峰]\xxqg'
    num = 2
    while True:
        model, left = db.popurl()
        if num <= 0:
            break
        if model == None:
            num = num - 1
            print('线程{}:空'.format(index))
            time.sleep(2)
            continue
        print('线程{},{}待处理:[{}]{}'.format(index, left,model.id, model.url))

        if model.filetype==2:
            if model.savepath != '':
                downfile(model.url,model.savepath)
            if model.savepath[-15:-3] == model.fromurl[-17:-5]:
                downjsdata(model)

        elif model.filetype ==1:
            ztspider.html(model)
        if os.path.isfile(model.savepath):
            db.setstatus(model.id, 1)
        else:
            db.setstatus(model.id, 10)

def downdata():
    basedir = r'\\192.168.10.60\d$\chuanshu\数据采集[张俊峰]\xxqg'
    db = urls()
    db.create_table()
    ztspider.index()

    print('down start')
    threadnum = 50
    p = ThreadPoolExecutor(max_workers=threadnum)
    obj_l = []
    for i in range(threadnum):
        obj = p.submit(run, i)
        obj_l.append(obj)
    p.shutdown()  # 等同于p.close(),p.join()
    print('down end')
    print('over')

def main():
    try:
        t = threading.Thread(target=downdata)
        t.setDaemon(True)
        t.start()
        t.join(60 * 60)
    except:
        traceback.print_exc()
    while True:
        n = 2*60*60
        while n>0:
            h = int(n/3600)
            m = int(n%3600/60)
            s = n%60
            lefttime = '{}小时{}分钟{}秒'.format(h,m,s)
            print('离下次更新还剩{}'.format(lefttime))
            time.sleep(1)
            n = n-1

if __name__ == '__main__':
    '''
    db = urls()
    db.create_table()
    ztspider.index()
    model = db.get(urls.id==13)
    db.setstatus(model.id,0)
    model = db.get(urls.id==14)
    db.setstatus(model.id,0)
    #downfile(model.url,model.savepath)
    #downjsdata(model)
    #run(1)
    #exit()
    '''
    main()